{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.58244264,  0.        ], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('MountainCarContinuous-v0', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, _, info = self.env.step(action)\n",
    "\n",
    "        reward = -1.0\n",
    "        if done:\n",
    "            reward = 100\n",
    "\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 400:\n",
    "            done = True\n",
    "\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAht0lEQVR4nO3df0wUd/4/8OfssrssP3b5vQsC1VaEIoryQ1hBW5VKPdtqba53jX80vUsbW7zUtmlSkm9tLrmE5i653PWusX9cru0fl9p4OXutsbQICiKL/BJdUbH2VLCwoODu8mPZX/P6/mGZj+tPQGR2l9cjmURm3rv72lnnmZn3zLxHICICY4zJQCF3AYyx+YsDiDEmGw4gxphsOIAYY7LhAGKMyYYDiDEmGw4gxphsOIAYY7LhAGKMyYYDiDEmG9kC6OOPP8bChQsRHh6OoqIitLS0yFUKY0wmsgTQl19+ibfffhsffPABOjo6kJubi/LycgwODspRDmNMJoIcN6MWFRWhsLAQf//73wEAoigiLS0Nv/vd7/Dee+/NdTmMMZmEzfUHut1utLe3o7KyUpqnUChQVlYGs9l8x9e4XC64XC7pb1EUMTw8jPj4eAiC8NBrZoxNDxFhZGQEKSkpUCjufqA15wF07do1+Hw+GAwGv/kGgwHnzp2742uqqqrw+9//fi7KY4zNot7eXqSmpt51+ZwH0ExUVlbi7bfflv622+1IT09Hb28vdDqdjJUxxu7E4XAgLS0N0dHR92w35wGUkJAApVKJgYEBv/kDAwMwGo13fI1Go4FGo7ltvk6n4wBiLIDdr4tkzs+CqdVq5Ofno7a2VponiiJqa2thMpnmuhzGmIxkOQR7++238fLLL6OgoACrVq3CX/7yF4yNjeGVV16RoxzGmExkCaBf/epXuHr1Knbv3g2r1YoVK1agurr6to5pxlhok+U6oAflcDig1+tht9u5D4ixADTVbTQozoIxxh4+t/sn+HzXoVKlQamMABD20K+z4wBijAEAbLav0Nf3/xAWloTw8CxERORCq12O8PClUKkMUCr1mDxvNVvBxAHEGPuZCJ/PBp/PBpfrPOz2rwGEQaEIh1qdBo1mMSIjV0GnK0dkZOGsfCIHEGPsLhQIC4tBWFgytNql0h6RSpUya5/AAcQYAwB4PGEID18Brfbxn8NmJTSaRVCpUqBQhANQzHqfEAcQYwwA0NGRiNWr/4u4uAUQBOWcfCaPiMgYAwB4PAoIQtSchQ/AAcQYkxEHEGNMNhxAjDHZcAAxxmTDAcQYkw0HEGNMNhxAjDHZcAAxxmTDAcQYkw0HEGNMNnwvGGPsvm4eOJWI/G5KfZAbVDmAGGN+iAgOhwOjo6Po7+/H1atXMTExAbvdDq1Wi9HRUWi1WqhUKoSHhyMuLg5RUVFYsGABoqOjoVarp/xZHECMzXNEBI/Hg+HhYXz33XcQRRFOpxNGoxGJiYlYvHgxYmNjoVKp/B6zLIoiJiYmMDw8jP7+fly8eBF2ux2RkZGIjY2d0mdzADE2DxERRFFEf38/Tpw4gWvXriE8PBz5+flIT0/3exDovQ6x9Ho9DAYDsrKypHk2mw3d3d1TqoMDiLF5xuVy4fz582hra4PRaEReXh6SkpIQFjbzQehvfl1sbCyys7On9DoOIMbmgcnDLIvFgtOnTyMrKwu//OUvERUVJWtdHECMhTifz4euri5YLBY8/vjjeOmll6BSqR76I3emggOIsRBFRPjpp5/Q2tqKhIQEbN26FREREQERPJM4gBgLQW63Gy0tLRgYGMD69esRExMTUMEziQOIsRBCROjr68Phw4dRUFAAk8kEpXLuxnieLg4gxkKEz+dDZ2cnBgYG8NxzzyE6Ojog93puNu17wRoaGvDss88iJSUFgiDgq6++8ltORNi9ezeSk5Oh1WpRVlaGH374wa/N8PAwtm/fDp1Oh5iYGPz2t7/F6OjoA30RxuazsbExfPfddxAEAeXl5dDpdAEfPsAMAmhsbAy5ubn4+OOP77j8j3/8Iz766CN88sknOH78OCIjI1FeXo6JiQmpzfbt29HV1YWamhocOHAADQ0NeO2112b+LRibp4gIvb29OHToEFauXImVK1cG9CHXbegBAKD9+/dLf4uiSEajkf70pz9J82w2G2k0Gvriiy+IiOjMmTMEgFpbW6U23377LQmCQD/99NOUPtdutxMAstvtD1I+Y0FNFEU6d+4c/fvf/6bR0VESRVHukiRT3UZndTiOixcvwmq1oqysTJqn1+tRVFQEs9kMADCbzYiJiUFBQYHUpqysDAqFAsePH7/j+7pcLjgcDr+JsfnM6/XCbDbj6tWreO655xAZGRkUh1y3mtUAslqtAACDweA332AwSMusViuSkpL8loeFhSEuLk5qc6uqqiro9XppSktLm82yGQsqHo8HR48ehUqlQklJCVQqldwlzVhQDEhWWVkJu90uTb29vXKXxNicIyJMTEygvr4e6enpKCgoCMq9npvNagAZjUYAwMDAgN/8gYEBaZnRaMTg4KDfcq/Xi+HhYanNrTQaDXQ6nd/E2HxCRHC5XKipqUFmZiYeffTRoA8fYJYDaNGiRTAajaitrZXmORwOHD9+HCaTCQBgMplgs9nQ3t4utamrq4MoiigqKprNchgLGWNjY6irq0NxcTFSU1NDInyAGVyIODo6igsXLkh/X7x4EZ2dnYiLi0N6ejp27dqFP/zhD8jIyMCiRYvw/vvvIyUlBVu3bgUAPP7443j66afx6quv4pNPPoHH48HOnTvx61//GikpKbP2xRgLBUSE0dFR1NbWYu3atYiLi5O7pNk13dNrhw8fJgC3TS+//DIR3Tg1+P7775PBYCCNRkMbNmyg7u5uv/cYGhqil156iaKiokin09Err7xCIyMjU66BT8Oz+cLhcNC3335L169fD6jT7Pcz1W1UILpptOkg4XA4oNfrYbfbuT+IhSQigs1mw7Fjx1BaWoqYmBi5S5qWqW6jQXEWjLH5hIgwPj6OY8eOwWQyQa/Xy13SQ8M3ozIWYCYmJnDkyBGsXr069Pp8bsF7QIwFEJfLhdraWhQUFEz5yRLBjAOIsQAxeYXzihUrkJSUFDKn2u+FA4ixAODxeFBbW4tHH30UCxYsmBfhA3AAMSY7IkJLSwuMRiMWLVo0b8IH4ABiTFZEBIvFArVajdzc3HkVPgAHEGOyISKcPn0afX19yM/Pn3fhA3AAMSab/v5+9PX1Yf369X7PXJ9P5ue3Zkxmkzdkr127Fmq1Wu5yZMMBxNgcczqdOHjwIEwmE8LDw+UuR1ZBfSV0EN7GxuY5n8+HhoYGPPHEE0hISJC7HNkF9R6Qw+HgEGJBQxRFmM1mpKam8tAzPwvqAGpqaoLb7Za7DMbui4hw9uxZuN1uZGdnz8szXncS1AGUlZWF2tpaeL1euUth7K6ICAMDA+jt7cWaNWs4fG4S1AG0cOFCJCQkoLOzkw/FWMAaHR2F2WzG2rVrg/oJFg9DUAeQIAjIz8+Hw+HA5cuXOYRYwPF4PDhy5AgKCwuh1WrlLifgBHUAAYBSqURpaSna2tpw/fp1ucthTCKKIpqbm7FkyZJ5dYPpdAR9AAGASqXCk08+iaNHj8LlcsldDmPSbRYKhQIZGRkcPncREgEkCALi4+OxbNkyNDU1QRRFuUti8xgR4erVq7h48SKKiorm7W0WUxEya0YQBCxatAiRkZE4ceIE9wcx2TidTjQ3N2P9+vUICwvqa30fupAJIOBGCOXl5cFqtcJqtXIIsTnn9XpRV1eHvLw8REdHy11OwAupAAKAsLAwrFu3DsePH4fT6ZS7HDaPEBHa2tqQmJiIBQsWyF1OUAi5AAIArVaLwsJCHDp0CB6PR+5y2DxARLhy5QpGRkZQUFDAnc5TFJIBJAgCUlJSkJKSwv1BbE44HA60tLRgzZo1UCqVcpcTNEIygIAbIbRy5UqMjIygp6dH7nJYCPN6vaitrUVRURE0Go3c5QSVkA0g4MZFiiUlJWhra8Pw8DDvCbFZN9nvs2TJEqSmpvKh1zSFdAABgEajgclkQn19PXw+n9zlsBDz448/Ynx8HNnZ2XKXEpSmFUBVVVUoLCxEdHQ0kpKSsHXrVnR3d/u1mZiYQEVFBeLj4xEVFYUXXngBAwMDfm16enqwefNmREREICkpCe++++5Du6NdEAQkJycjIyMDbW1tvBfEZo3NZkNLSwuKi4v5YsMZmtZaq6+vR0VFBZqbm1FTUwOPx4ONGzdibGxMavPWW2/hm2++wb59+1BfX4++vj5s27ZNWu7z+bB582a43W40NTXh888/x2effYbdu3fP3re6hSAIWLp0KUZHR/Hjjz9yCLEH5vF40NjYiKeeegoRERFylxO86AEMDg4SAKqvryciIpvNRiqVivbt2ye1OXv2LAEgs9lMREQHDx4khUJBVqtVarNnzx7S6XTkcrmm9Ll2u50AkN1un1a94+Pj9K9//YscDse0XsfYzURRpGPHjlFXVxeJoih3OQFpqtvoA+032u12AEBcXBwAoL29HR6PB2VlZVKbrKwspKenw2w2AwDMZjOWLVsGg8EgtSkvL4fD4UBXV9cdP8flcsHhcPhNM6HValFWVoaGhgYexIzNCBHhhx9+wNDQELKysrjT+QHNOIBEUcSuXbtQUlKCnJwcAIDVaoVarUZMTIxfW4PBAKvVKrW5OXwml08uu5Oqqiro9XppSktLm2nZSExMRFpaGvcHsRkZGRnBmTNn8NRTT3G/zyyY8RqsqKjA6dOnsXfv3tms544qKytht9ulqbe3d8bvJQgCcnJycPXqVVy6dIlDiE2Z1+tFQ0MDSkpK5v3jdGbLjAJo586dOHDgAA4fPozU1FRpvtFohNvths1m82s/MDAAo9Eotbn1rNjk35NtbqXRaKDT6fymB6FQKFBWVoZTp05hdHT0gd6LzQ9EhIaGBqSkpPDjdGbRtAKIiLBz507s378fdXV1WLRokd/y/Px8qFQq1NbWSvO6u7vR09MDk8kEADCZTLBYLBgcHJTa1NTUQKfTzem1FFqtFkVFRWhsbOT+IHZPRISenh54vV6sWLGC+31m03R6tl9//XXS6/V05MgR6u/vl6bx8XGpzY4dOyg9PZ3q6uqora2NTCYTmUwmabnX66WcnBzauHEjdXZ2UnV1NSUmJlJlZeWs97DfjyiK1N7eTseOHeOzGeyuRkZG6D//+Q85nU65SwkaU91GpxVAAO44ffrpp1Ibp9NJb7zxBsXGxlJERAQ9//zz1N/f7/c+ly5dok2bNpFWq6WEhAR65513yOPxzPqXmwqv10vff/899fT0PPB7sdDjdrtp37591NfXJ3cpQWWq26hAFHy9sA6HA3q9Hna7/YH7g4AbI9hVV1dj48aNiIiI4F1sBuDGoVd7e7s00B3/v5i6qW6jfB4RQHh4OFatWoVDhw7xeNJMcuXKFVy/fp37fR4iDiD83/hBqampOHnyJJ+aZ3A4HDh+/DhKSkp4fJ+HiAPoZ4IgYMWKFRgYGEB/f7/c5TAZ+Xw+NDY2ori4mB8m+JBxAN1EqVRi7dq1aGxs9LvBls0fRITOzk4kJyfzwwTnAAfQLSIiIqTrg7g/aH4hIvT29uLq1atYvnw5h88c4AC6hSAISE9PR3x8PCwWC/cHzSNOpxPHjx/ncZ3nEAfQHUyOJ33lyhX89NNPHELzgM/nQ0NDA1avXo3IyEi5y5k3OIDuQqlUorS0FC0tLfx8sRBHRDh58iSSkpKQkpIidznzCgfQPej1ehQWFqKxsZHHkw5RRIS+vj5cuXIFubm53O8zxziA7iM1NRXx8fHo6OjgQ7EQ5HQ60draig0bNvD4PjLgNX4fgiAgNzcXVqsVfX19HEIhxOv14ujRo1i1ahUiIyN570cGHEBTEBYWhg0bNnB/UAihn5/nFR8fj+TkZLnLmbc4gKZocvygI0eO8PhBIaC3txejo6NYuXIl7/nIiANoiiafL5aYmMjPmw9iRISRkRF0dHSgtLSUr/eRGQfQNEwOy+BwOPh580HK5/Ph4MGDKC4u5ue4BwAOoGmafN58Z2cnrl+/zntCQUQURbS0tGDZsmUwGo186BUAOIBmQKPRoKSkBPX19dwfFCSICBcuXIDX60VWVpbc5bCfcQDNgCAIiI+Px9KlS2E2m/mm1QBHRBgcHERbWxs/xz3A8C8xQ4IgYPHixVAqlejq6uJDsQDmcrnQ3NyMZ555Bmq1Wu5y2E04gB6AIAhYtWoVzp8/z4OYBSifz4ejR48iLy8P0dHRcpfDbsEB9IBUKhWefvpptLa28kMOA8zkoPIxMTFITU3lTucAxAE0CyIiImAymVBfXw+PxyN3OQw3wud///sfnE4nP9EigHEAzQJBEJCYmIiFCxeiubmZO6VlRkQYGhrCmTNnsHr1ar7YMIBxAM0SQRCQnZ0NjUbDndIyczqd+O6771BaWoqwsDC5y2H3wAE0iwRBQEFBAXp7e3Hp0iUOIRl4vV40NDRg/fr1iI2N5UOvAMcBNMsUCgWeeOIJnDhxgq+UnmOiKKKpqQkLFy6E0WiUuxw2BRxAD0FkZCTKysrQ2NiIiYkJucuZF4gIFosFSqUSmZmZvOcTJKYVQHv27MHy5cuh0+mg0+lgMpnw7bffSssnJiZQUVGB+Ph4REVF4YUXXsDAwIDfe/T09GDz5s2IiIhAUlIS3n333ZC8nSE6Ohr5+fmorq6G2+2Wu5yQRkS4dOkShoeHUVxczOETRKYVQKmpqfjwww/R3t6OtrY2rF+/Hlu2bEFXVxcA4K233sI333yDffv2ob6+Hn19fdi2bZv0ep/Ph82bN8PtdqOpqQmff/45PvvsM+zevXt2v1UAmHzc8+OPP46mpiY+M/aQEBGsViu6urr4McrBiB5QbGws/eMf/yCbzUYqlYr27dsnLTt79iwBILPZTEREBw8eJIVCQVarVWqzZ88e0ul05HK5pvyZdrudAJDdbn/Q8h86URSpo6OD2traSBRFucsJOQ6Hg/773//S6Oio3KWwm0x1G51xH5DP58PevXsxNjYGk8mE9vZ2eDwelJWVSW2ysrKQnp4Os9kMADCbzVi2bBkMBoPUpry8HA6HQ9qLCjWTY0oPDw/j1KlT3Ck9i8bHx3H06FGsWbOGn+UVpKYdQBaLBVFRUdBoNNixYwf279+P7OxsWK1WqNVqxMTE+LU3GAywWq0AAKvV6hc+k8snl92Ny+WCw+Hwm4KJQqHAunXrMDw8zKfnZ4nL5UJ1dTVyc3Nv+z/Hgse0AygzMxOdnZ04fvw4Xn/9dbz88ss4c+bMw6hNUlVVBb1eL01paWkP9fMehrCwMJSWluLs2bOwWq0cQg/A5/OhsbEReXl5WLBgAXc6B7FpB5BarcbixYuRn5+Pqqoq5Obm4q9//SuMRiPcbjdsNptf+4GBAemaDKPReNtZscm/73XdRmVlJex2uzT19vZOt+yAoFKp8OSTT6KtrQ1DQ0NylxOUJsMnNTUVjzzyiNzlsAf0wNcBiaIIl8uF/Px8qFQq1NbWSsu6u7vR09MDk8kEADCZTLBYLBgcHJTa1NTUQKfTITs7+66fodFopFP/k1OwioiIwLp169Da2orr16/LXU5QEUURbW1t0Gq1WLJkCe/5hILp9Gy/9957VF9fTxcvXqRTp07Re++9R4Ig0Pfff09ERDt27KD09HSqq6ujtrY2MplMZDKZpNd7vV7KycmhjRs3UmdnJ1VXV1NiYiJVVlY+lB72QCWKItlsNtq7dy8NDw/z2bEp8Pl81NbWRp2dneTz+eQuh93HVLfRaQXQb37zG3rkkUdIrVZTYmIibdiwQQofIiKn00lvvPEGxcbGUkREBD3//PPU39/v9x6XLl2iTZs2kVarpYSEBHrnnXfI4/FMp4ygD6BJQ0NDVF1dTQ6HQ+5SAprP56OmpiY6duwYh0+QmOo2KhAFX2+ow+GAXq+H3W4P6sMxALh+/TqOHj2K0tJSvnnyDkRRhMVigdvtRn5+Po/nHCSmuo3yrymzmJgYmEwmNDU1YWRkRO5yAoooimhubobH4+HwCVH8i8pscjCzkpISNDU1YXh4mE/R40b4dHR0AADy8vI4fEIU/6oBIiYmBkVFRTh06NC8P0Xv8/nQ3NwMlUrFj9EJcfzLBghBEBAbG4tNmzahtbUV/f3983JPyOPxoL6+HhqNBsuXL+fwCXH86waYqKgorFu3Dh0dHbhw4cIDhRDdOMsZNEE2NjaGuro6LFy4kAeSnyd4wNwAIwgCwsPDsXHjRpjNZoyPj2PZsmXT3hMQRRF//vOfcfnyZWRnZyMjIwOLFi1CYmIiwsPDoVKpAmYDJyI4HA7U1NRg1apVSEtLC5ja2MPFp+EDmM/nQ0tLC5xOJ9asWQOVSjXl146NjeHpp59GY2MjgBtXk0dHR8NgMCAjIwNLlizB0qVLkZmZieTkZCQmJkKj0Ugb/lwFABHh8uXLOHXqFEpLSxEXFzcnn8serqluoxxAAY6I8OOPP+L8+fMoLS1FdHT0lMJhcHAQOTk5uHr16l3bCIIAlUqFhIQEGAwGZGZmIicnB5mZmVi8eDEyMzOh1Wpn8+v4EUURXV1dOH/+PDZv3uwXgCy4TXUb5UOwACcIAh577DHodDpUV1ejuLh4SocoQ0ND931IIhHB7Xajr68PfX19OHHiBIAbd+5rtVp8+eWX2LRp06x9l5s/1+VyobGxETqdDlu2bOHH58xT3AkdBCavFXruuefQ3d2NxsbG+46j3d7ePuNxk7xeL8LDw5GRkTGj198LEWFwcBAHDhxASkoKCgsLOXzmMf7lg8Rk5/SGDRvQ3t6Or7/+GmvXrkV8fPxte0NEBJvNBlEUkZycjBUrViA6Oho9PT04efIknE7nfT9Pp9PdNnjcg/J4PDCbzbDZbCgvL0dUVBQfcs1zHEBBRqFQoLCwEIsXL0ZdXR2SkpJQXFyMsLAwaWMmIjQ1NSE1NRUvvvii1G80OUTuV199JT2pQ6vVIjIyEm63GyMjI9Ip+zVr1sxa/w8Rob+/Hy0tLUhNTeXB45mEAyhIxcTEYMuWLTh16hS+/vprFBQUIC0tTTpdPzo6ivLycr9Oa4VCgaysLKxYsQItLS3IzMzE+vXrERcXh4mJCXR2dqKhoQFerxdLly6dlZAYHx9Ha2sr+vr6sGnTJuj1et7rYRIOoCAlCALCwsKwcuVKLF68GI2NjbBYLCguLkZ8fDz+9re/4YsvvoDL5fJ7nUKhgMFgQHJyMp555hlER0cDuDFa4+rVqzExMYHW1laUlJTMOCiICF6vFydPnsT58+dRUFCA0tJS3utht+EACnKCIECn02HTpk0YHBxETU0NoqOjkZubC51Od8fT8Fu3bkVRUdFtZ8mUSiUKCgpw5coVJCcnz6gel8sFi8WCCxcu4LHHHsO2bdv49Dq7Kw6gECEIAgwGA1588UWpv8Vms8Hn80GhUEgBsGDBArz44ouora1FZ2fnbe+j0+mwfft2LFiwYMqfTUQYGRmBxWJBb28vUlNT8eyzzyIiIoKDh90TB1CIUSqVWLBgAbZu3Ypr167h3LlzOHXqFIaGhpCSkoItW7YgMjISS5YsgcVigc/n83t9Xl4enn322XseLk12VDudTly4cAFnzpyBQqFARkYGtm3bFlC3ebDAxldChzgigsfjwaVLl9DR0QGXy4WEhAQYjUZcvnwZFosFwI2+ofT0dGzbtk3qF7r5PbxeL1wuF4aHh3H58mVcuHABarUaaWlpyMrKQkJCAt+5ziR8JTQDcOPQTK1WY8mSJcjIyIDb7cbg4CAuXLgAn88HtVoNm80GpVIJtVqNY8eOSRci2mw2KBQKjI2NQRRFhIeHIzIyEhkZGVi6dCliYmI4dNgD4QCaRwRBgEajQVpaGtLS0kBEEEURoihiZGQETqcTHo8HLpcLarUaRASdToeoqCjpsGpyYmw2cADNY4IgQKlUQqlU8l3oTBa8/8wYkw0HEGNMNhxAjDHZcAAxxmTDAcQYkw0HEGNMNhxAjDHZcAAxxmTDAcQYkw0HEGNMNkF5K8bkDfwzfeoDY+zhmtw27zfYRlAG0NDQEAAgLS1N5koYY/cyMjICvV5/1+VBGUCTN0729PTc88sxfw6HA2lpaejt7eVxlKaI19nMTI6SmZKScs92QRlAk2PQ6PV6/k8xAzqdjtfbNPE6m76p7BxwJzRjTDYcQIwx2QRlAGk0GnzwwQfQaDRylxJUeL1NH6+zhysoB6VnjIWGoNwDYoyFBg4gxphsOIAYY7LhAGKMySYoA+jjjz/GwoULER4ejqKiIrS0tMhdkmyqqqpQWFiI6OhoJCUlYevWreju7vZrMzExgYqKCsTHxyMqKgovvPACBgYG/Nr09PRg8+bNiIiIQFJSEt599114vd65/Cqy+fDDDyEIAnbt2iXN43U2RyjI7N27l9RqNf3zn/+krq4uevXVVykmJoYGBgbkLk0W5eXl9Omnn9Lp06eps7OTfvGLX1B6ejqNjo5KbXbs2EFpaWlUW1tLbW1tVFxcTKtXr5aWe71eysnJobKyMjpx4gQdPHiQEhISqLKyUo6vNKdaWlpo4cKFtHz5cnrzzTel+bzO5kbQBdCqVauooqJC+tvn81FKSgpVVVXJWFXgGBwcJABUX19PREQ2m41UKhXt27dPanP27FkCQGazmYiIDh48SAqFgqxWq9Rmz549pNPpyOVyze0XmEMjIyOUkZFBNTU19MQTT0gBxOts7gTVIZjb7UZ7ezvKysqkeQqFAmVlZTCbzTJWFjjsdjuA/7tht729HR6Px2+dZWVlIT09XVpnZrMZy5Ytg8FgkNqUl5fD4XCgq6trDqufWxUVFdi8ebPfugF4nc2loLoZ9dq1a/D5fH4/OgAYDAacO3dOpqoChyiK2LVrF0pKSpCTkwMAsFqtUKvViImJ8WtrMBhgtVqlNndap5PLQtHevXvR0dGB1tbW25bxOps7QRVA7N4qKipw+vRpNDY2yl1KQOvt7cWbb76JmpoahIeHy13OvBZUh2AJCQlQKpW3nY0YGBiA0WiUqarAsHPnThw4cACHDx9GamqqNN9oNMLtdsNms/m1v3mdGY3GO67TyWWhpr29HYODg8jLy0NYWBjCwsJQX1+Pjz76CGFhYTAYDLzO5khQBZBarUZ+fj5qa2uleaIoora2FiaTScbK5ENE2LlzJ/bv34+6ujosWrTIb3l+fj5UKpXfOuvu7kZPT4+0zkwmEywWCwYHB6U2NTU10Ol0yM7OnpsvMoc2bNgAi8WCzs5OaSooKMD27dulf/M6myNy94JP1969e0mj0dBnn31GZ86coddee41iYmL8zkbMJ6+//jrp9Xo6cuQI9ff3S9P4+LjUZseOHZSenk51dXXU1tZGJpOJTCaTtHzylPLGjRups7OTqqurKTExcV6dUr75LBgRr7O5EnQBRET0t7/9jdLT00mtVtOqVauoublZ7pJkA+CO06effiq1cTqd9MYbb1BsbCxFRETQ888/T/39/X7vc+nSJdq0aRNptVpKSEigd955hzwezxx/G/ncGkC8zuYGD8fBGJNNUPUBMcZCCwcQY0w2HECMMdlwADHGZMMBxBiTDQcQY0w2HECMMdlwADHGZMMBxBiTDQcQY0w2HECMMdlwADHGZPP/Aau1jyyn/gflAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box([-1.2  -0.07], [0.6  0.07], (2,), float32)\n",
      "env.action_space= Box(-1.0, 1.0, (1,), float32)\n",
      "state= [-0.44963592  0.        ]\n",
      "action= [-0.6594274]\n",
      "next_state= [-0.45117524 -0.00153932]\n",
      "reward= -1.0\n",
      "done= False\n",
      "info= {}\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.td3.td3.TD3 at 0x7f66d4cec880>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import TD3\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = TD3(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=1),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    buffer_size=100_0000,\n",
    "    learning_starts=400,\n",
    "    batch_size=400,\n",
    "    gamma=0.99,\n",
    "    train_freq=(1, 'episode'),\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-400.0, 0.0)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #训练\n",
    "# model.learn(20_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/8.TD3.Mountain Car Continuous')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cdaaf12a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(31.4, 1.9339079605813716)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = TD3.load('save/8.TD3.Mountain Car Continuous')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "414e1478",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAhhUlEQVR4nO3da1Bbdf4/8PcJuREgCfeUklhsqdB7yzWlVtsiWHG16u6sjg8cd1enSnesOs7IzN86O7MzOO6TXXfd+mBn1Qc7253ubNe106IUWiglXALFUlrRaltSIYEWknAJIcn5/B8o+ZW2ViiUk8vnNXNmzDnfJJ8cet6e8z3nfI9ARATGGJOATOoCGGPRiwOIMSYZDiDGmGQ4gBhjkuEAYoxJhgOIMSYZDiDGmGQ4gBhjkuEAYoxJhgOIMSYZyQLo/fffx7Jly6BWq1FUVIS2tjapSmGMSUSSAPrXv/6F1157DW+//TY6Ozuxfv16lJeXY3BwUIpyGGMSEaS4GbWoqAgFBQX4y1/+AgAQRRFGoxG//e1v8eabby52OYwxicgX+wunpqbQ0dGBqqqq4DyZTIbS0lJYLJZbvsfr9cLr9QZfi6KI4eFhJCcnQxCEu14zY2xuiAijo6PIyMiATPbjB1qLHkBXr15FIBBAenr6jPnp6en48ssvb/me6upq/O53v1uM8hhjC8hmsyEzM/NHly96AN2JqqoqvPbaa8HXLpcLJpMJNpsNWq1WwsoYY7fidrthNBqRkJBw23aLHkApKSmIiYmBw+GYMd/hcMBgMNzyPSqVCiqV6qb5Wq2WA4ixEPZTXSSLfhZMqVQiLy8PdXV1wXmiKKKurg5ms3mxy2GMSUiSQ7DXXnsNzz33HPLz81FYWIg//vGPGB8fx/PPPy9FOYwxiUgSQL/85S8xNDSEffv2wW63Y8OGDaipqbmpY5oxFtkkuQ5ovtxuN3Q6HVwuF/cBMRaCZruNhsVZMMbY3Tc19R0CgREoFEbExGgAyO/6dXYcQIwxAIDT+V/09/8/yOVpUKtzoNGsR2zsOqjVq6FQpCMmRofp81YLFUwcQIyxH4gIBJwIBJzwer+Cy/U/AHLIZGoolUaoVCsQF1cIrbYccXEFC/KNHECMsR8hg1yuh1y+BLGxq4N7RApFxoJ9AwcQYwwA4PPJoVZvQGxs7g9hsxEqVRYUigzIZGoAsgXvE+IAYowBADo7U7F58ydISloKQYhZlO/kAGKMAQB8PhkEIX5G+IiiiMnJSUxMTGB8fBx2ux1XrlyB3+/Hz3/+c8TEzC+oOIAYi1KiKCIQCCAQCGBychIOhwNWqxWTk5O4fPkyLl26hL6+PjgcDjgcDgwNDcHr9cLn8+Hee+9FeXk59Hr9vGrgAGIsCo2MjODdd9/Ft99+C5vNBrvdjuHhYXi9XkxOTv7k+51OJ0ZGRjiAGGNz53Q68de//hVut/uO3j80NIT+/n5kZWXNqw5+KgZjUSg1NRW5ubl3/H4igiiK866D94AYi0JxcXFITU2ddfukpCRkZmYiEAigr68PY2NjuHr16rzr4ABiLEplZ2cD+P62CoPBgOTkZGRkZMBms+Hrr7+G3+8HAKxcuRIVFRXB0Q2vXr2KTz75BN988828a+AAYixKbdy4EQCwfPly7Nq1C319fcjKykJhYSGam5tx/PhxJCYm4pFHHoFWqw1ehJiamoqKigpcvnx53jVwADEWhQRBgF6vR1xcHMrLyxEfHx9cJpfLUVxcjMuXLyM+Ph46nW7GFdCCIGDJkiUwmUzzroMDiLEolZ2dDaPRiMTExJuWKZVKlJWVQaPRQCaT4cZhw+RyOcrLy+ddA58FYyxKGY1GPP3009BoNDctEwQBv/nNb/D2228jI+Pmm0+zs7ORlpY27xo4gBiLUnFxcXjrrbdQUlJy08MDs7KykJubi4SEBDz66KMwmUyQy+VQKBRYuXIlysrKIJfLb9ozmis+BGMsislkMhQXF0Mul6O/vx96vR65ubnYuHEjhoeHMTAwgKGhIcTGxkKhUCAuLg5utxtHjx6FQqGAWq1GUlIS4uPjsXTpUiQkJECpVM76+zmAGItyCoUCeXl5OHPmDNRqNdxuNz7//HMYDAakpqZixYoVSExMhEKhmLGnNH2j6nRQXbx4ES6XC3FxcbfsV7oVDiDGotD0lcwDAwM4ffo0rl69ivj4eBQWFsJkMs14EOjtxgDS6XRIT09HTk5OcJ7T6URvb++s6uAAYizKeL1efPXVV7BarTAYDNi0aRPS0tIgl9/5IPTXvy8xMRGrVq2a1fs4gBiLAkQEn8+H7u5unD17Fjk5OfjFL34x4/ofKXAAMRbhAoEAenp60N3djdzcXDzzzDNQKBR3/ZE7s8EBxFiEIiJ89913aG9vR0pKCnbt2gWNRhMSwTONA4ixCDQ1NYW2tjY4HA5s374der0+pIJnGgcQYxGEiNDf34/jx48jPz8fZrN53uM2300cQIxFiEAggK6uLjgcDjz22GNISEgIyb2e6835VozGxkb87Gc/Q0ZGBgRBwH//+98Zy4kI+/btw5IlSxAbG4vS0lJ8/fXXM9oMDw/j2WefhVarhV6vx69//WuMjY3N64cwFs3Gx8fx2WefQRAElJeXzxg+I5TNOYDGx8exfv16vP/++7dc/u677+K9997DBx98gNbW1uDt/tcPdP3ss8+ip6cHtbW1OHz4MBobG/Hiiy/e+a9gLEoREWw2G44dO4aNGzdi48aNIX3IdROaBwB06NCh4GtRFMlgMNAf/vCH4Dyn00kqlYr++c9/EhHRuXPnCAC1t7cH2xw9epQEQaDvvvtuVt/rcrkIALlcrvmUz1hYE0WRvvzyS/r3v/9NY2NjJIqi1CUFzXYbXdC74S9evAi73Y7S0tLgPJ1Oh6KiIlgsFgCAxWKBXq9Hfn5+sE1paSlkMhlaW1tv+blerxdut3vGxFg08/v9sFgsGBoawmOPPYa4uLiwOOS60YIGkN1uBwCkp6fPmJ+enh5cZrfbbxpHRC6XIykpKdjmRtXV1dDpdMHJaDQuZNmMhRWfz4eTJ09CoVCgpKQECoVC6pLuWFiMB1RVVQWXyxWcbDab1CUxtuiICJOTk2hoaIDJZEJ+fn5Y7vVcb0EDyGAwAAAcDseM+Q6HI7jMYDBgcHBwxnK/34/h4eFgmxupVCpotdoZE2PRhIjg9XpRW1uL++67D/fee2/Yhw+wwAGUlZUFg8GAurq64Dy3243W1laYzWYAgNlshtPpREdHR7BNfX09RFFEUVHRQpbDWMQYHx9HfX09iouLkZmZGRHhA9zBhYhjY2O4cOFC8PXFixfR1dWFpKQkmEwm7N27F7///e+RnZ2NrKwsvPXWW8jIyMCuXbsAALm5uXj44Yfxwgsv4IMPPoDP58OePXvw9NNP33LsWcaiGRFhbGwMdXV12Lp1K5KSkqQuaWHN9fTa8ePHCcBN03PPPUdE358afOuttyg9PZ1UKhXt2LGDent7Z3zGtWvX6JlnnqH4+HjSarX0/PPP0+jo6Kxr4NPwLFq43W46evQojYyMhNRp9p8y221UIJrnqNIScLvd0Ol0cLlc3B/EIhIRwel04tSpU9iyZQv0er3UJc3JbLfRsDgLxlg0ISJMTEzg1KlTMJvN0Ol0Upd01/DNqIyFmMnJSZw4cQKbN2+OvD6fG/AeEGMhxOv1oq6uDvn5+bN+skQ44wBiLERMX+G8YcMGpKWlRcyp9tvhAGIsBPh8PtTV1eHee+/F0qVLoyJ8AA4gxiRHRGhra4PBYEBWVlbUhA/AAcSYpIgI3d3dUCqVWL9+fVSFD8ABxJhkiAhnz55Ff38/8vLyoi58AA4gxiQzMDCA/v5+bN++fcYz16NJdP5qxiQ2fUP21q1boVQqpS5HMhxAjC0yj8eDI0eOwGw2Q61WS12OpML6SugwvI2NRblAIIDGxkY88MADSElJkbocyYX1HpDb7eYQYmFDFEVYLBZkZmby0DM/COsAam5uxtTUlNRlMPaTiAjnz5/H1NQUVq1aFZVnvG4lrAMoJycHdXV18Pv9UpfC2I8iIjgcDthsNtx///0cPtcJ6wBatmwZUlJS0NXVxYdiLGSNjY3BYrFg69atYf0Ei7shrANIEATk5eXB7Xbj8uXLHEIs5Ph8Ppw4cQIFBQWIjY2VupyQE9YBBAAxMTHYsmULrFYrRkZGpC6HsSBRFNHS0oKVK1dG1Q2mcxH2AQQACoUCDz74IE6ePAmv1yt1OYwFb7OQyWTIzs7m8PkRERFAgiAgOTkZa9euRXNzM0RRlLokFsWICENDQ7h48SKKioqi9jaL2YiYNSMIArKyshAXF4fTp09zfxCTjMfjQUtLC7Zv3w65PKyv9b3rIiaAgO9DaNOmTbDb7bDb7RxCbNH5/X7U19dj06ZNSEhIkLqckBdRAQQAcrkc27ZtQ2trKzwej9TlsChCRLBarUhNTcXSpUulLicsRFwAAUBsbCwKCgpw7Ngx+Hw+qcthUYCIcOXKFYyOjiI/P587nWcpIgNIEARkZGQgIyOD+4PYonC73Whra8P999+PmJgYqcsJGxEZQMD3IbRx40aMjo6ir69P6nJYBPP7/airq0NRURFUKpXU5YSViA0g4PuLFEtKSmC1WjE8PMx7QmzBTff7rFy5EpmZmXzoNUcRHUAAoFKpYDab0dDQgEAgIHU5LMJ88803mJiYwKpVq6QuJSzNKYCqq6tRUFCAhIQEpKWlYdeuXejt7Z3RZnJyEpWVlUhOTkZ8fDyeeuopOByOGW36+vpQUVEBjUaDtLQ0vPHGG3ftjnZBELBkyRJkZ2fDarXyXhBbME6nE21tbSguLuaLDe/QnNZaQ0MDKisr0dLSgtraWvh8PpSVlWF8fDzY5tVXX8Wnn36KgwcPoqGhAf39/XjyySeDywOBACoqKjA1NYXm5mZ8/PHH+Oijj7Bv376F+1U3EAQBq1evxtjYGL755hsOITZvPp8PTU1NeOihh6DRaKQuJ3zRPAwODhIAamhoICIip9NJCoWCDh48GGxz/vx5AkAWi4WIiI4cOUIymYzsdnuwzf79+0mr1ZLX653V97pcLgJALpdrTvVOTEzQP/7xD3K73XN6H2PXE0WRTp06RT09PSSKotTlhKTZbqPz2m90uVwAgKSkJABAR0cHfD4fSktLg21ycnJgMplgsVgAABaLBWvXrkV6enqwTXl5OdxuN3p6em75PV6vF263e8Z0J2JjY1FaWorGxkYexIzdESLC119/jWvXriEnJ4c7nefpjgNIFEXs3bsXJSUlWLNmDQDAbrdDqVRCr9fPaJueng673R5sc334TC+fXnYr1dXV0Ol0wcloNN5p2UhNTYXRaOT+IHZHRkdHce7cOTz00EPc77MA7ngNVlZW4uzZszhw4MBC1nNLVVVVcLlcwclms93xZwmCgDVr1mBoaAiXLl3iEGKz5vf70djYiJKSkqh/nM5CuaMA2rNnDw4fPozjx48jMzMzON9gMGBqagpOp3NGe4fDAYPBEGxz41mx6dfTbW6kUqmg1WpnTPMhk8lQWlqKM2fOYGxsbF6fxaIDEaGxsREZGRn8OJ0FNKcAIiLs2bMHhw4dQn19PbKysmYsz8vLg0KhQF1dXXBeb28v+vr6YDabAQBmsxnd3d0YHBwMtqmtrYVWq13UayliY2NRVFSEpqYm7g9it0VE6Ovrg9/vx4YNG7jfZyHNpWf7pZdeIp1ORydOnKCBgYHgNDExEWyze/duMplMVF9fT1arlcxmM5nN5uByv99Pa9asobKyMurq6qKamhpKTU2lqqqqBe9h/ymiKFJHRwedOnWKz2awHzU6Okr/+c9/yOPxSF1K2JjtNjqnAAJwy+nDDz8MtvF4PPTyyy9TYmIiaTQaeuKJJ2hgYGDG51y6dIl27txJsbGxlJKSQq+//jr5fL4F/3Gz4ff76fPPP6e+vr55fxaLPFNTU3Tw4EHq7++XupSwMtttVCAKv15Yt9sNnU4Hl8s17/4g4PsR7GpqalBWVgaNRsO72AzA94deHR0dwYHu+N/F7M12G+XziADUajUKCwtx7NgxHk+aBV25cgUjIyPc73MXcQDh/8YPyszMxBdffMGn5hncbjdaW1tRUlLC4/vcRRxAPxAEARs2bIDD4cDAwIDU5TAJBQIBNDU1obi4mB8meJdxAF0nJiYGW7duRVNT04wbbFn0ICJ0dXVhyZIl/DDBRcABdAONRhO8Poj7g6ILEcFms2FoaAjr1q3j8FkEHEA3EAQBJpMJycnJ6O7u5v6gKOLxeNDa2srjOi8iDqBbmB5P+sqVK/juu+84hKJAIBBAY2MjNm/ejLi4OKnLiRocQD8iJiYGW7ZsQVtbGz9fLMIREb744gukpaUhIyND6nKiCgfQbeh0OhQUFKCpqYnHk45QRIT+/n5cuXIF69ev536fRcYB9BMyMzORnJyMzs5OPhSLQB6PB+3t7dixYweP7yMBXuM/QRAErF+/Hna7Hf39/RxCEcTv9+PkyZMoLCxEXFwc7/1IgANoFuRyOXbs2MH9QRGEfnieV3JyMpYsWSJ1OVGLA2iWpscPOnHiBI8fFAFsNhvGxsawceNG3vOREAfQLE0/Xyw1NZWfNx/GiAijo6Po7OzEli1b+HofiXEAzcH0sAxut5ufNx+mAoEAjhw5guLiYn6OewjgAJqj6efNd3V1YWRkhPeEwogoimhra8PatWthMBj40CsEcADdAZVKhZKSEjQ0NHB/UJggIly4cAF+vx85OTlSl8N+wAF0BwRBQHJyMlavXg2LxcI3rYY4IsLg4CCsVis/xz3E8F/iDgmCgBUrViAmJgY9PT18KBbCvF4vWlpa8Oijj0KpVEpdDrsOB9A8CIKAwsJCfPXVVzyIWYgKBAI4efIkNm3ahISEBKnLYTfgAJonhUKBhx9+GO3t7fyQwxAzPai8Xq9HZmYmdzqHIA6gBaDRaGA2m9HQ0ACfzyd1OQzfh8+3334Lj8fDT7QIYRxAC0AQBKSmpmLZsmVoaWnhTmmJERGuXbuGc+fOYfPmzXyxYQjjAFoggiBg1apVUKlU3CktMY/Hg88++wxbtmyBXC6Xuhx2GxxAC0gQBOTn58Nms+HSpUscQhLw+/1obGzE9u3bkZiYyIdeIY4DaIHJZDI88MADOH36NF8pvchEUURzczOWLVsGg8EgdTlsFjiA7oK4uDiUlpaiqakJk5OTUpcTFYgI3d3diImJwX333cd7PmFiTgG0f/9+rFu3DlqtFlqtFmazGUePHg0un5ycRGVlJZKTkxEfH4+nnnoKDodjxmf09fWhoqICGo0GaWlpeOONNyLydoaEhATk5eWhpqYGU1NTUpcT0YgIly5dwvDwMIqLizl8wsicAigzMxPvvPMOOjo6YLVasX37djz++OPo6ekBALz66qv49NNPcfDgQTQ0NKC/vx9PPvlk8P2BQAAVFRWYmppCc3MzPv74Y3z00UfYt2/fwv6qEDD9uOfc3Fw0NzfzmbG7hIhgt9vR09PDj1EORzRPiYmJ9Le//Y2cTicpFAo6ePBgcNn58+cJAFksFiIiOnLkCMlkMrLb7cE2+/fvJ61WS16vd9bf6XK5CAC5XK75ln/XiaJInZ2dZLVaSRRFqcuJOG63mz755BMaGxuTuhR2ndluo3fcBxQIBHDgwAGMj4/DbDajo6MDPp8PpaWlwTY5OTkwmUywWCwAAIvFgrVr1yI9PT3Ypry8HG63O7gXFWmmx5QeHh7GmTNnuFN6AU1MTODkyZO4//77+VleYWrOAdTd3Y34+HioVCrs3r0bhw4dwqpVq2C326FUKqHX62e0T09Ph91uBwDY7fYZ4TO9fHrZj/F6vXC73TOmcCKTybBt2zYMDw/z6fkF4vV6UVNTg/Xr19/0b46FjzkH0H333Yeuri60trbipZdewnPPPYdz587djdqCqqurodPpgpPRaLyr33c3yOVybNmyBefPn4fdbucQmodAIICmpiZs2rQJS5cu5U7nMDbnAFIqlVixYgXy8vJQXV2N9evX409/+hMMBgOmpqbgdDpntHc4HMFrMgwGw01nxaZf3+66jaqqKrhcruBks9nmWnZIUCgUePDBB2G1WnHt2jWpywlL0+GTmZmJe+65R+py2DzN+zogURTh9XqRl5cHhUKBurq64LLe3l709fXBbDYDAMxmM7q7uzE4OBhsU1tbC61Wi1WrVv3od6hUquCp/+kpXGk0Gmzbtg3t7e0YGRmRupywIooirFYrYmNjsXLlSt7ziQRz6dl+8803qaGhgS5evEhnzpyhN998kwRBoM8//5yIiHbv3k0mk4nq6+vJarWS2Wwms9kcfL/f76c1a9ZQWVkZdXV1UU1NDaWmplJVVdVd6WEPVaIoktPppAMHDtDw8DCfHZuFQCBAVquVurq6KBAISF0O+wmz3UbnFEC/+tWv6J577iGlUkmpqam0Y8eOYPgQEXk8Hnr55ZcpMTGRNBoNPfHEEzQwMDDjMy5dukQ7d+6k2NhYSklJoddff518Pt9cygj7AJp27do1qqmpIbfbLXUpIS0QCFBzczOdOnWKwydMzHYbFYjCrzfU7XZDp9PB5XKF9eEYAIyMjODkyZPYsmUL3zx5C6Iooru7G1NTU8jLy+PxnMPEbLdR/mtKTK/Xw2w2o7m5GaOjo1KXE1JEUURLSwt8Ph+HT4Tiv6jEpgczKykpQXNzM4aHh/kUPb4Pn87OTgDApk2bOHwiFP9VQ4Rer0dRURGOHTsW9afoA4EAWlpaoFAo+DE6EY7/siFCEAQkJiZi586daG9vx8DAQFTuCfl8PjQ0NEClUmHdunUcPhGO/7ohJj4+Htu2bUNnZycuXLgQVSE0Pj6O+vp6LFu2jAeSjxIcQCFGEASo1WqUlZVhYGAAZ86cifihPIgILpcLR48eRW5uLrKysjh8ogQHUIhSKBQoKSnBxMQETpw4EbGP+yEiXL58GQ0NDdi+fTtMJhOHTxThAAphMTExKC4uhslkQm1tLdxud0QdkomiiLNnz6KjowNlZWVITEyUuiS2yDiAQpwgCFi+fDny8/NRU1MDm80W9iFERJicnER9fT08Hg8ef/xxqNVq3vOJQhxAYWD6WqHHHnsMvb29aGpqCttxtIkIg4ODOHz4MDIyMlBQUMDP7opiHEBhYrpzeseOHVCr1fjf//6Hq1evhtXekM/nw8mTJ9Ha2ory8nLk5ubyXk+U4//1hBmZTIaCggKsWLEC9fX1SEtLQ3FxMeRyechuzESEgYEBtLW1ITMzkwePZ0F8M2qYIiIEAgGcOXMGFy9eRH5+PoxGY8hduDcxMYH29nb09/dj586d0Ol0IRuUbOHMdhvlAApzRITR0VE0NTVBFEUUFxcjOTlZ0o2ciOD3+/HFF1/gq6++Qn5+PpYvX857PVGEAyjKTHfu1tfXIyEhAUVFRUhJSVn0IPJ6veju7saFCxewfPlyrF27FiqVivd6ogwHUJQKBALB/ha/34/CwkIsXbr0rvYRTe+FdXd3w2azITMzExs3boRGo+HgiVIcQFGMvh/pEkNDQ8G9kdzcXKxcuRJpaWmQyWTzCobpfzIejwcXLlzAuXPnIJPJkJ2djdWrV0OhUHDwRDkOIAbg+7Dwer3o6+tDV1cXPB4PUlJSkJGRgXvuuQexsbHBQ6QfC43pPh2v14vh4WFcvnwZFy5cgFKphNFoRE5ODlJSUkKuA5xJhwOI3YSIMDU1hcHBQVy4cAEjIyOw2+2Ij4+HIAhISEiAWq2G3++HWq2G0+mETCbD+Pg4RFGEWq1GXFwcsrOzkZqaCr1ez6HDbmm22yhfBxRFBEGASqWC0WiE0WgEEUEURYiiiNHRUXg8Hvh8Pni9XiiVShARtFot4uPjg4dVt9tTYmyuOICimCAIiImJQUxMDJKSkqQuh0Uh3n9mjEmGA4gxJhkOIMaYZDiAGGOS4QBijEmGA4gxJhkOIMaYZDiAGGOS4QBijEmGA4gxJpmwvBVj+v5Zt9stcSWMsVuZ3jZ/6l73sAyga9euAQCMRqPElTDGbmd0dBQ6ne5Hl4dlAE3fONnX13fbH8dmcrvdMBqNsNlsPIzJLPE6uzPTo2RmZGTctl1YBtD0GDQ6nY7/UdwBrVbL622OeJ3N3Wx2DrgTmjEmGQ4gxphkwjKAVCoV3n77bahUKqlLCSu83uaO19ndFZZjQjPGIkNY7gExxiIDBxBjTDIcQIwxyXAAMcYkE5YB9P7772PZsmVQq9UoKipCW1ub1CVJprq6GgUFBUhISEBaWhp27dqF3t7eGW0mJydRWVmJ5ORkxMfH46mnnoLD4ZjRpq+vDxUVFdBoNEhLS8Mbb7wBv9+/mD9FMu+88w4EQcDevXuD83idLRIKMwcOHCClUkl///vfqaenh1544QXS6/XkcDikLk0S5eXl9OGHH9LZs2epq6uLHnnkETKZTDQ2NhZss3v3bjIajVRXV0dWq5WKi4tp8+bNweV+v5/WrFlDpaWldPr0aTpy5AilpKRQVVWVFD9pUbW1tdGyZcto3bp19MorrwTn8zpbHGEXQIWFhVRZWRl8HQgEKCMjg6qrqyWsKnQMDg4SAGpoaCAiIqfTSQqFgg4ePBhsc/78eQJAFouFiIiOHDlCMpmM7HZ7sM3+/ftJq9WS1+td3B+wiEZHRyk7O5tqa2vpgQceCAYQr7PFE1aHYFNTU+jo6EBpaWlwnkwmQ2lpKSwWi4SVhQ6XywXg/27Y7ejogM/nm7HOcnJyYDKZguvMYrFg7dq1SE9PD7YpLy+H2+1GT0/PIla/uCorK1FRUTFj3QC8zhZTWN2MevXqVQQCgRl/dABIT0/Hl19+KVFVoUMURezduxclJSVYs2YNAMBut0OpVEKv189om56eDrvdHmxzq3U6vSwSHThwAJ2dnWhvb79pGa+zxRNWAcRur7KyEmfPnkVTU5PUpYQ0m82GV155BbW1tVCr1VKXE9XC6hAsJSUFMTExN52NcDgcMBgMElUVGvbs2YPDhw/j+PHjyMzMDM43GAyYmpqC0+mc0f76dWYwGG65TqeXRZqOjg4MDg5i06ZNkMvlkMvlaGhowHvvvQe5XI709HReZ4skrAJIqVQiLy8PdXV1wXmiKKKurg5ms1nCyqRDRNizZw8OHTqE+vp6ZGVlzViel5cHhUIxY5319vair68vuM7MZjO6u7sxODgYbFNbWwutVotVq1Ytzg9ZRDt27EB3dze6urqCU35+Pp599tngf/M6WyRS94LP1YEDB0ilUtFHH31E586doxdffJH0ev2MsxHR5KWXXiKdTkcnTpyggYGB4DQxMRFss3v3bjKZTFRfX09Wq5XMZjOZzebg8ulTymVlZdTV1UU1NTWUmpoaVaeUrz8LRsTrbLGEXQAREf35z38mk8lESqWSCgsLqaWlReqSJAPgltOHH34YbOPxeOjll1+mxMRE0mg09MQTT9DAwMCMz7l06RLt3LmTYmNjKSUlhV5//XXy+XyL/Gukc2MA8TpbHDwcB2NMMmHVB8QYiywcQIwxyXAAMcYkwwHEGJMMBxBjTDIcQIwxyXAAMcYkwwHEGJMMBxBjTDIcQIwxyXAAMcYkwwHEGJPM/wdTKNxF4p2eUgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31.0 70 [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, 100]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 5 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
